import csv
import os

import requests
from lxml import etree

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.149 Safari/537.36"
}
r = requests.get('http://27.223.1.57:10000/PythonApplication/index.aspx?oneClassGuid=171030103404278262',
                 headers=headers)
html = etree.HTML(r.text)
tr_list = html.xpath('//tbody/tr/@onclick')
td_list = html.xpath('//tbody/tr/td[@class="twoNavigationSize"]/text()')
dict = {}

for xd, item in enumerate(tr_list):
    # print(item)
    index = item.rfind(',')
    # print(index)
    # print(item[index+2:-2])
    dict[td_list[xd]] = item[index + 2:-2]


# print(dict)
def main():
    for name, id in dict.items():
        for i in range(1, 6):
            print("爬取%s,第%s页" % (name, i))
            url = 'http://27.223.1.57:10000/PythonApplication/webbasesite/dataInfoList.aspx?lkocok_pageNo=%s&oneClassGuid=%s' % (
                i, id)
            r = requests.get(url)
            html = etree.HTML(r.text)
            # 序号
            x_id = html.xpath('//tbody/tr/td[@width="50"]/text()')
            # # ['序号\n                                ', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13','14', '15']
            x_id.pop(0)
            # 诉求内容列表
            content_list = html.xpath('//tbody/tr/td[@align="left"]/@title')
            date_list = html.xpath('//tbody/tr/td[@width="100"]/text()')
            # print(date_list[::2])
            # print(date_list[1::2])
            # 诉求时间列表
            Appeal_time = date_list[::2]
            # 回复时间列表
            Reply_time = date_list[1::2]
            list = []
            for num, item in enumerate(x_id):
                temp = {
                    'id': x_id[num],
                    'content': content_list[num],
                    'Appeal_time': Appeal_time[num],
                    'Reply_time': Reply_time[num]
                }
                list.append(temp)
            # print(list)

            download_path = os.getcwd() + '/青岛问政/'
            if not os.path.exists(download_path):
                os.makedirs(download_path)

            with open('./青岛问政/%s' % name + '.csv', 'a', encoding='utf-8')as f:
                f_csv = csv.DictWriter(f, ['id', 'content', 'Appeal_time', 'Reply_time', ])
                f_csv.writeheader()
                f_csv.writerows(list)
                print("保存信息到CSV....ok")


qd = main()
